# _*_coding:utf-8_*_
import urllib
import urllib.request
import re
import threading
import hotel


class Spider:
    def __init__(self):
        self.siteURL = 'http://www.tripadvisor.cn/Hotels-g294217-Hong_Kong-Hotels.html'
        self.baseURL = 'http://www.tripadvisor.cn'

    def getEachPageHotelUrls(self, URL):
        # 收集每个页面的酒店主页地址
        u = urllib.request.URLopener()  # Python 3: urllib.request.URLOpener
        u.addheaders = []
        u.addheader('User-Agent', 'Opera/9.80 (Windows NT 6.1; WOW64; U; de) Presto/2.10.289 Version/12.01')
        u.addheader('Accept-Language', 'de-DE,de;q=0.9,en;q=0.8')
        u.addheader('Accept',
                    'text/html, application/xml;q=0.9, application/xhtml+xml, image/png, image/webp, image/jpeg, image/gif, image/x-xbitmap, */*;q=0.1')
        try:
            # 如果页面找不到,则异常中断
            response = u.open(URL)
        except Exception as e:
            print(e)
            return
        html = response.read().decode('utf-8')
        response.close()

        pattern = re.compile(
            # 匹配酒店地址的正则表达式
            '<a target="_blank" href="(.*?)".*?class="property_title "', re.S)
        items = re.findall(pattern, html)
        return items;

    def getAllPageUrls(self):
        # 收集全部的酒店主页地址
        # @ /Hotels-g294217-oaXX-Hong_Kong-Hotels.html#ACCOM_OVERVIEW 固定的地址格式
        # 所有的酒店URL
        allHotelUrls = [];
        # 第一页的URL特殊
        url_1 = self.siteURL;
        allHotelUrls += self.getEachPageHotelUrls(url_1);
        # 其他页面需要遍历URL查找
        for i in range(24, 100):
            url_s = self.baseURL + "/Hotels-g294217-oa" + str((i - 1) * 30) + "-Hong_Kong-Hotels.html#ACCOM_OVERVIEW"
            tempUrls = self.getEachPageHotelUrls(url_s)
            if not tempUrls:
                break;
            allHotelUrls += tempUrls
        # 最后的所有酒店的地址全部保存在allHotelUrls数组变量中
        for item in allHotelUrls:
            print(item)

    def getHotelInfo(self, URL):
        # 收集单个酒店信息
        u = urllib.request.URLopener()  # Python 3: urllib.request.URLOpener
        u.addheaders = []
        u.addheader('User-Agent', 'Opera/9.80 (Windows NT 6.1; WOW64; U; de) Presto/2.10.289 Version/12.01')
        u.addheader('Accept-Language', 'de-DE,de;q=0.9,en;q=0.8')
        u.addheader('Accept',
                    'text/html, application/xml;q=0.9, application/xhtml+xml, image/png, image/webp, image/jpeg, image/gif, image/x-xbitmap, */*;q=0.1')
        try:
            # 如果页面找不到,则异常中断
            response = u.open(URL)
        except Exception as e:
            print(e)
            return
        html = response.read().decode('utf-8')
        # 将html文本中的单引号转换为双引号,方便后面的处理
        html = html.replace('\'', '"')
        html = html.replace('\n', '')
        response.close()

        pattern = re.compile(
            # 匹配酒店名字的正则表达式
            '<div class="heading_height"></div>(.*?)<span class="altHead">(.*?)</span>',
            re.S)
        items = re.findall(pattern, html)
        item = items[0]
        hb = hotel.Hotel()
        hb.setZName(item[0].strip('\n'))
        hb.setEName(item[1].strip('\n'))

        #  '.*?property="reviewCount" content="(.*?)"'
        # 匹配酒店评论数的正则表达式
        pattern1 = re.compile('href="#REVIEWS" property="reviewCount" content="(.*?)"', re.S)
        items1 = re.findall(pattern1, html)
        item1 = items1[0]
        hb.setPlNum(int(item1))

        # <b class="rank">排名第3</b>
        # 匹配排名的正则表达式
        pattern2 = re.compile('<b class="rank">排名第(.*?)</b>', re.S)
        items2 = re.findall(pattern2, html)
        item2 = items2[0]
        hb.setrank(int(item2))

        # <div class="fl">.*?<script>.*?</script>(.*?)</div>
        # 匹配电话的正则表达式
        pattern3 = re.compile('function escramble_(.*?)document', re.S)
        items3 = re.findall(pattern3, html)
        scr = items3[0]
        scr = scr.replace(' ', '')
        pattern3_ = re.compile('"(.*?)"', re.S)
        numbers = re.findall(pattern3_, scr)
        # print(numbers)
        hb.setrank(numbers[0] + numbers[2] + numbers[4] + numbers[1] + numbers[3])

        # 匹配地址,前缀是"中国香港"
        pattern4 = re.compile('<span class="street-address" property="streetAddress">(.*?)</span>', re.S)
        items4 = re.findall(pattern4, html)
        # print(items4)
        item4 = items4[0]
        hb.setAddress('中国香港' + item4)

        # 匹配轮播图 由于网站做了限制,不能获取真实数据
        # pattern5 = re.compile('<div class="sizedThumb_container"><img.*?src=".*?"', re.S)
        # items5 = re.findall(pattern5, html)
        # print(items5)

        # 匹配评论
        pattern6 = re.compile('<div id="review_.*?" class="reviewSelector.*?(<.*?>.*?</.*?>)*?</div>', re.S)
        items6 = re.findall(pattern6, html)
        print(items6)

        print(hb.tostring())


spider = Spider()
# spider.getEachPageHotelUrls(spider.siteURL)
# spider.getAllPageUrls()
# /Hotel_Review-g294217-d548045-Reviews-Four_Seasons_Hotel_Hong_Kong-Hong_Kong.html
spider.getHotelInfo(
    spider.baseURL + '/Hotel_Review-g294217-d548045-Reviews-Four_Seasons_Hotel_Hong_Kong-Hong_Kong.html')
